#a fifo-queue use disk space to store  items
#use pickle to save item 
# @author devdoer@gmail.com
import cPickle
import os
from os import path
class DiskQue:
	def __init__(self,**keys):
		#parse args
		key=keys['name']
		if keys.has_key('buf_size'):
			bufSize=keys['buf_size']
		else:
			bufSize=10000

		if keys.has_key('back_dir'):
			backDir=keys['back_dir']
		else:
			backDir='./'

		self.BUF_SIZE=bufSize
		self.BACK_DIR=backDir
		self.BACK_FILE_SIZE=self.IN_BUF_SIZE=self.OUT_BUF_SIZE=self.BUF_SIZE
		self.queKey=key
		try:
			
			os.mkdir(self.BACK_DIR)
		except 	OSError:
			pass

		try:
			os.mkdir(path.join(self.BACK_DIR,self.queKey))
		except 	OSError:
			pass

		self.__len=0
		self.__headBuf=[]
		self.__headBufIdx=0
		self.__outBuf=[]
		self.__headBackFileID=0
		self.__tailBackFileID=0
	def sync(self):

		#save out-buf
		f=self.__gen_dat_file('outbuf','wb')
		cPickle.dump(self.__outBuf,f,1)
		f.close()


		#save other meta data
		metaFile=self.__gen_dat_file('meta','wt')
		#1.size
		metaFile.write(str(self.__len)+'\n')
		#2.head back file id
		metaFile.write(str(self.__headBackFileID)+'\n')
		#3.head back file start idx 
		metaFile.write(str(self.__headBufIdx)+'\n')
		#4.tail back file id
		metaFile.write(str(self.__tailBackFileID)+'\n')
		metaFile.close()

		#ok,status is consistent
		#remove un-used back files
		#all file whose id is lesser than checkpointed head-back-file id
		#remove(f)
		


	def load(self):
		#load meta informaton
		metaFile=self.__gen_dat_file('meta','rt')
		#1.size
		line=metaFile.readline()
		self.__len=int(line)
		#2.head back file id
		line=metaFile.readline()
		self.__headBackFileID=int(line)
		#3.head  back file start idx
		line=metaFile.readline()
		self.__headBufIdx=int(line)
		#4.tail back file id
		line=metaFile.readline()
		self.__tailBackFileID=int(line)
		metaFile=self.__gen_dat_file('meta','rt')


		#read out-buf dat
		f=self.__gen_dat_file('outbuf','rb')
		self.__outBuf=cPickle.load(f)
		f.close()

		#read head-buf from head  back disk file
		if self.__headBackFileID!=self.__tailBackFileID:
			f=self.__gen_back_file(self.__headBackFileID,'rb')
			self.__headBuf=cPickle.load(f)
			f.close()
		

	def __gen_dat_file(self,name,mode):
		return open(path.join(self.BACK_DIR,self.queKey,name+'.dat'),mode)
	def __gen_back_file(self,id,mode):
		return open(path.join(self.BACK_DIR,self.queKey,str(id)+'.dqb'),mode)
	def append(self,item):
		self.__outBuf.append(item)
		self.__len+=1

		if len(self.__outBuf)>=self.BUF_SIZE:
			#save outbuf as new disk file
			self.__flush_outbuf()

			self.sync()

	def __pop_headbuf(self):
	#pop one item from headbuf	
		item=self.__headBuf[self.__headBufIdx]
		self.__headBufIdx+=1
		self.__len-=1
		return item
	def __flush_outbuf(self):
	#store outbuf as new disk file
		f=self.__gen_back_file(self.__tailBackFileID,'wb')
		cPickle.dump(self.__outBuf,f,1)
		f.close()
		self.__tailBackFileID+=1
		#new outbuf
		self.__outBuf=[]

	def pop(self):
		#if self.__len<=0:
		#	raise IndexError('pop from empty queue') 
		try:
			item=self.__pop_headbuf()
		except IndexError :
		#headbuf is empty,use next disk file
			dist=self.__headBackFileID-self.__tailBackFileID
			if dist==0 or dist==-1:
			#disk queue is empty or become empty,get  from out-buf
				if self.__outBuf:
					
					#trick: replace outbuf as new headbuf
					self.__headBuf=self.__outBuf
					if dist==-1:
					#queue become empty,use next head id	
						self.__headBackFileID+=1
					self.__headBufIdx=0

					#store outbuf,then it will be the head back file
					self.__flush_outbuf()

				else:
					raise IndexError('pop from empty queue')
			else :	
				#read  head-buf from next disk file
				self.__headBackFileID+=1
				f=self.__gen_back_file(self.__headBackFileID,'rb')
				self.__headBuf=cPickle.load(f)
				self.__headBufIdx=0
				f.close()

			#new headbuf is ready
			item=self.__pop_headbuf()
			self.sync()
		return item

	def __len__(self):
		return self.__len
class Item:
	def __init__(self,u):
		self.u=u
def test():
	import random		
	que=DiskQue(name='test')	
	que.append(Item('hello'))
	for i in xrange(3000):	
		item=Item(str(random.randint(0,10000)))
		que.append(item)
	que.sync()	
	que.load()	
	assert len(que)==3001
	
	assert que.pop().u=='hello'
	i=0
	while True:
		try:	
			que.pop()
			i+=1
		except IndexError:
			break
	assert i==3000	
	que.sync()

